import re
import os
from typing import Union, List

# 预处理类，批量将故事文本去重整理
class ProcessFile:
    def __init__(self, file_name: Union[str, List[str]]):
        self.files = []
        if isinstance(file_name, str):
            self.files.append(file_name)
        elif isinstance(file_name, list):
            self.files.extend(file_name)

    def __my_private_split_story(self, text):
        # 分割文本成行
        lines = text.splitlines()

        # 去除单个字母并以.结尾的行，同时处理行内重复的内容
        unique_lines = []
        seen_lines = set()

        for line in lines:
            # 去除单个字母并以.结尾的行
            if re.match(r'^[A-Za-z]\.$', line.strip()):
                continue

            # 去除行内重复的短语
            phrases = line.split(',')
            unique_phrases = []
            seen_phrases = set()

            for phrase in phrases:
                cleaned_phrase = phrase.strip()
                if cleaned_phrase not in seen_phrases:
                    unique_phrases.append(cleaned_phrase)
                    seen_phrases.add(cleaned_phrase)

            unique_line = ', '.join(unique_phrases)

            # 添加到去重后的行列表中
            if unique_line not in seen_lines:
                unique_lines.append(unique_line)
                seen_lines.add(unique_line)

        # 返回处理后的文本
        return '\n'.join(unique_lines)

    def __my_private_process_story(self, file_name):
        source = os.path.join('./story', file_name + '.txt')
        target = os.path.join('./split_story', file_name + '_split.txt')
        # 处理文本
        with open(source, 'r', encoding='utf-8') as f:
            data = f.read()
            processed_text = self.__my_private_split_story(data)  # 假设有一个名为 split_story 的函数处理文本

        # 存储处理后的文本到新文件中
        with open(target, 'w', encoding='utf-8') as f:
            f.write(processed_text)

    def process(self):
        for file in self.files:
            self.__my_private_process_story(file)


if __name__ == '__main__':
    files = ['story1','story2']
    ps = ProcessFile(files)
    ps.process()